In [60]:
import numpy as np
from scipy.special import expit
In [61]:
# Define NN class
class NeuralNetwork(object):
# Initialize NN
def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
# Nodes
self.in_nodes = input_nodes
self.hn_nodes = hidden_nodes
self.ot_nodes = output_nodes
# Learning rate
self.learning_rate = learning_rate
# Weight Matix
## Weight, input layer to hidden layer
## np.random.normal(center, standard_deviation, dimension)
self.wih = np.random.normal(0.0, pow(self.hn_nodes, -0.5), (self.hn_nodes, self.in_nodes))
## Weight, hidden layer to output layer
self.who = np.random.normal(0.0, pow(self.ot_nodes, -0.5), (self.ot_nodes, self.hn_nodes))
# Activation function
self.activation_function = lambda x: expit(x)
# Training logic
def train(self, input_list, target_list):
# FeedForward
## Transform list to 2-dimensional transposed ndarray(to column vector)
inputs = np.array(input_list, ndmin=2).T
targets = np.array(target_list, ndmin=2).T
## Operate input signal to hidden layer
hidden_inputs = np.dot(self.wih, inputs)
## Operate ouput signal from hidden layer
hidden_outputs = self.activation_function(hidden_inputs)
## Operate hidden signal to output layer
output_inputs = np.dot(self.who, hidden_outputs)
## Operate output
output_outputs = self.activation_function(output_inputs)
# Back Propagation
## Errors
output_errors = targets - output_outputs
## Hidden layers' errors
hidden_errors = np.dot(self.who.T, output_errors)
## Update weight between output and hidden layers
self.who += self.learning_rate * np.dot((output_errors * output_outputs * (1-output_outputs)),
np.transpose(hidden_outputs))
## Update weight between hidden and input layers
self.wih += self.learning_rate * np.dot((hidden_errors * hidden_outputs * (1-hidden_outputs)),
np.transpose(inputs))
def query(self, input_list):
# Transform list to 2-dimensional transposed ndarray(to column vector)
inputs = np.array(input_list, ndmin=2).T
# Input to hidden layer
hidden_inputs = np.dot(self.wih, inputs)
# Outputs from hidden layer
hidden_outputs = self.activation_function(hidden_inputs)
# Hidden to output layer
output_inputs = np.dot(self.who, hidden_outputs)
# Outputs from output layer
output_outputs = self.activation_function(output_inputs)
return output_outputs
In [62]:
# Set initial values
input_nodes = 3
hidden_nodes = 3
output_nodes = 3
learning_rate = 0.3
n = NeuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)
In [63]:
# Check the class
n.query([1.2, -0.5, 3.7])
Out[63]:
In [64]:
# Load Training dataset
with open('./dataframe/[HYStudy 23th] mnist_train_100.csv', 'r') as f:
train_list = f.readlines()
In [65]:
# Label, 28*28 matrix
train_list[0]
Out[65]:
In [66]:
# Import library to display data
import matplotlib.pylab as plt # to use imshow
In [67]:
# Display data
## Create Subplots
fig, axes = plt.subplots(2, 5, subplot_kw={"xticks": [], "yticks": []})
# axes.flat: returns the axes as 1-dimensional(flat) array
for ax, i in zip(axes.flat, range(0, 10)):
all_values = train_list[i].split(',')
# String to float, all_values[0] = label, trainsform to 28*28 matrix
image_array = np.asfarray(all_values[1:]).reshape(28, 28)
ax.imshow(image_array, cmap="Greys")
ax.set_title(i)
fig.subplots_adjust(hspace=-0.3)
plt.show()
In [68]:
# Display "1"(with grid)
all_values = train_list[3].split(',')
# String to float, all_values[0] = label, transform to 28*28 matrix
image_array = np.asfarray(all_values[1:]).reshape(28, 28)
# Draw image
plt.imshow(image_array, cmap="Greys")
plt.grid(True)
plt.title("Label {}".format(all_values[0]))
plt.show()
train_list[3]
Out[68]:
In [69]:
# Normalize RGB values
# +0.01 to prevent '0' as output
# '0' input makes activation function's output to '0'
norm_input = (np.asfarray(all_values[1:]) / 255 * 0.99) + 0.01
print(norm_input)
In [70]:
# Create output label array to put in NN
# Range must be 0 < X < 1 (Sigmoid)
out_nodes = 10
# Target value(label)
# 1 -> 0.99, 0 -> 0.01 to satisfy constraint
targets = np.zeros(out_nodes) + 0.01
# all_values[0] = label
targets[int(all_values[0])] = 0.99
# label = 1(index[1] = 0.99)
targets
Out[70]:
In [100]:
# Define network structure
input_nodes = 784 # Data = 28*28
# The number of hidden nodes: choose heuristically
# Must be smaller than input nodes to extract core feature
# Properly larger than output nodes
hidden_nodes = 100
# The number of output nodes: same as the number of labels
output_nodes = 10
# Learning rate
learning_rate = 0.3
# Create instance of NN
n = NeuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)
In [121]:
# Train the NN
for record in train_list:
# Split the csv data by seperator ','
all_values = record.split(',')
# Normalize input values
inputs = (np.asfarray(all_values[1:]) / 255 * 0.99) + 0.01
# Create target value
targets = np.zeros(output_nodes) + 0.01
targets[int(all_values[0])] = 0.99
n.train(inputs, targets)
pass
In [73]:
# Load test dataset
with open('./dataframe/[HYStudy 23th] mnist_test_10.csv', 'r') as f:
test_list = f.readlines()
In [74]:
# Test 1st data
test_values = test_list[0].split(',')
print("Correct answer is", test_values[0])
In [122]:
# Send query with nomalized value
ans = n.query((np.asfarray(test_values[1:]) / 255.0 * 0.99) + 0.01)
print(ans)
print("Maximum value is {} and index is {}".format(max(ans), np.argmax(ans)))
print("Predicted label is", np.argmax(ans))